# -*- coding: utf-8 -*-
#import scrapy
from mycwpjt.items import MycwpjtItem
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class WeisuenSpider(CrawlSpider):
    name = 'weisuen'
    allowed_domains = ['sohu.com']
    start_urls = ['http://news.sohu.com/']

    rules = (
        #url "http://news.sohu.com/20160926/n469167364.shtml"
        #正则表达式提取为：'.*?/n.*?shtml'
        #Rule(LinkExtractor(allow=(r'https://news.cnblogs.com/n/page/\d',))),
        #Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=('.*?/n.*?shtml'),allow_domains=('sohu.com')),callback='parse_item',follow=True),
    )

    def parse_item(self, response):
        print("****begin to get data****")
        #i = {}
        #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
        #i['name'] = response.xpath('//div[@id="name"]').extract()
        #i['description'] = response.xpath('//div[@id="description"]').extract()
        i =MycwpjtItem()
        i["name"]=response.xpath("/html/head/title/text()").extract()
        i["link"]=response.xpath("//link[@rel='canonical']/@href").extract()
        print(i)
        yield i
